# !/bin/python

import numpy as np
from time import localtime, strftime
from scipy import sparse
import sys
from sklearn import decomposition
import pandas as pd
from math import sqrt
from functools import partial
from array import array
from scipy.sparse import csr_matrix,hstack,coo_matrix

#####
def now():
	return strftime("%Y-%m-%d %H:%M:%S", localtime())
def error(s):
	#print >> sys.stderr,now(), s
	print >> sys.stdout,now(), s
def write_file(filepath,mtx):
	error(' writing into %s'%filepath)
	indices = mtx.nonzero()
	row_indices = indices[0]
	col_indices = indices[1]
	vals = sparse.csr_matrix(mtx[row_indices,col_indices])
	vals = vals.toarray()[0]
	assert len(row_indices) == len(col_indices) == len(vals)
	fh = file(filepath,'wb')
	for i in xrange(len(vals)):
		fh.write("%d %d %f\n"%(row_indices[i],col_indices[i],vals[i]))
	fh.close()
	error(" writing done.")
#####
error("encoding...")

#python encoding_pca_nom.py $sys.csv $sys_type.csv $component
data = sys.argv[1]
print data
types = sys.argv[2]
print(types)
component = int(sys.argv[3])
# en
original_path= sys.argv[1]
original_type_path=sys.argv[2]

#original_path=r"./out_attr12_num_half_edges.csv"
#original_type_path=r"./out_attr_type_12_num_half_edges.csv"
type_dict = {}
item_csv_type = pd.read_csv(original_type_path)
for catagory in item_csv_type[:0]:
	if ((item_csv_type['%s'%(catagory)][0:].item() =='C') |( item_csv_type['%s'%(catagory)][0:].item() == 'MC')):
		type_dict['%s'%(catagory)]=str
item_csv = pd.read_csv(original_path,converters =type_dict,engine = 'python')

matrix_l = []

def split_str(string):
	strings =  string.lower().split(',')
	strings = [x.strip() for x in strings]
	return strings
def encode_table(ser):
	ser_vals = ser.values
	x = []
	for i in ser_vals:
		x.extend(i)
	ser_vals = pd.Series(x)
	table = ser_vals.value_counts()
	for	idx in range(table.shape[0]):
		table[idx] = idx
	return table, table.shape[0]

def encode_by_table(li,en_table):
	indices = [en_table[i] for i in li]
	return indices

def convert2matrix(ser):
	row = []
	col = []
	values = []
	for i in range(len(ser)):
		row= row+[i]*len(ser[i])
		col = col+ser[i]
		values = values +[1]*len(ser[i])	
	
	max_num = []
	for i in ser:
		if i != []:
			max_num.append(max(i))
	if (max_num == []):
		y = 1
	elif ( max_num != []):
		y=max(max_num)+1
		#print(y)
	matrix_tmp = coo_matrix((values,(row,col)),shape=(3884,y))
	return matrix_tmp

def encode_category(ser):
	ser = ser.fillna("unknown")
	ser = ser.apply(split_str)
	table, shape_col = encode_table(ser)
	encoder = partial(encode_by_table,en_table = table)
	ser = ser.apply(encoder)
	matrix_tmp = convert2matrix(ser)
	return matrix_tmp

#deal with encoded category
def split_str_num(string):
	#for i in range(len(string)):
	#	string[i] = str(string[i])
	if string !='':
		string = str(string)
		strings =  string.split(',')
		strings = [int(x.strip('\" \'')) for x in strings]
	elif string == '':
		strings = []	
	return strings

def convert2matrix_num(ser):
	row = []
	col = []
	values = []	
	for i in range(3883):
		row= row+[i]
		col = col+[0]
		values = values +[ser[i]]
	
	matrix_tmp = coo_matrix((values,(row,col)),shape=(3884,1))
	return matrix_tmp

def encode_category_num(ser):
	#ser = ser.fillna('u')
	ser = ser.replace('None','').replace('nan','')

	ser = ser.apply(split_str_num)
	matrix_tmp = convert2matrix(ser)
	return matrix_tmp

#deal with N
def scale(x,min_v,range_v):
	return (x-min_v)*1./range_v

def encode_numerical(ser):
	min_val = ser.min()
	range_val = ser.max() - min_val
	
	ser = ser.fillna(ser.mean())
	ser = ser.replace('None',ser.mean()).replace('nan',ser.mean())
	if (range_val == 0):
		matrix_tmp = coo_matrix((0),shape=(3884,0))
	elif (range_val !=0):
		scaler = partial(scale,min_v=min_val, range_v = range_val)
		ser = ser.apply(scaler)
		# print(ser)
		matrix_tmp = convert2matrix_num(ser)
	return matrix_tmp

#use to tell this category is encoded or not
def is_num(num):
	try:
		int(num)
		return True
	except ValueError:
		return False
		

in_size = item_csv.shape
	
for catagory in item_csv_type[:0]:
	item_csv_id_name = item_csv[['id','%s'%catagory]].fillna('None')

	if ((item_csv_type['%s'%(catagory)][0:].item() == 'C') |(item_csv_type['%s'%(catagory)][0:].item() == 'MC' )):
		for i in range(3884):
			tmp = item_csv_id_name['%s'%catagory][i]
			#print(tmp == 'test1')
			if ((tmp != 'None') & (tmp !='nan')):
				test = str(tmp.split(',')[0]).strip(' ').strip('\"\'')
				break

		print('%s'%catagory,test,is_num(test))
		if is_num(test) == True:
			matrix_tmp = encode_category_num(item_csv_id_name['%s'%catagory])
		elif is_num(test) == False:
			matrix_tmp = encode_category(item_csv_id_name['%s'%catagory])
		matrix_l.append(matrix_tmp)
		
 		
		
	elif item_csv_type['%s'%(catagory)][0:].item() == 'N':
		matrix_tmp = encode_numerical(item_csv_id_name['%s'%catagory])
		matrix_l.append(matrix_tmp)
#print(matrix_l)
for i in range(1,len(matrix_l)):
	print(i)
	if (matrix_l[i].getnnz() != 0):
		matrix_l[0] = hstack([matrix_l[0],matrix_l[i]])

# lyl
print 'size of the input data:',in_size
csc_matrix_l = matrix_l[0].tocsc()
shape_matrix = csc_matrix_l.shape
print 'shape of the matrix after encoding:',shape_matrix
write_file('attr_after_encoding',csc_matrix_l)
print(shape_matrix)
# lyl-end

# en-end

#def now():
#	return strftime("%Y-%m-%d %H:%M:%S", localtime())

#def error(s):
#	print >> sys.stderr,now(), s

def matrix_from_file(path,mtx_shape):
	error(' reading %s...'%path)
	edges = np.loadtxt(path)
	row_indices = edges[:,0]
	col_indices = edges[:,1]
	vals        = edges[:,2]
	error(' reading finish...')
	mtx = sparse.csr_matrix((vals,(row_indices,col_indices)), shape = mtx_shape,dtype=float)
	return mtx

#def write_file(filepath,mtx):
#	error(' writing into %s'%filepath)
#	indices = mtx.nonzero()
#	row_indices = indices[0]
#	col_indices = indices[1]
#	vals = sparse.csr_matrix(mtx[row_indices,col_indices])
#	vals = vals.toarray()[0]
#	assert len(row_indices) == len(col_indices) == len(vals)
#	fh = file(filepath,'wb')
#	for i in xrange(len(vals)):
#		fh.write("%d %d %f\n"%(row_indices[i],col_indices[i],vals[i]))
#	fh.close()
#	error(" writing done.")

# normalize
def normalize_col(x):
	mean = x.mean()
	sd = x.std()
	return (x-mean)*1.0/sd
	
row = int(shape_matrix[0])
column = int(shape_matrix[1])+1
mtx_shape = (row,column)
#mtx_shape = (3953,2001)
#row = 3953 # normalize

attr_sp_mtx = matrix_l[0]
# attr_sp_mtx = csr_matrix_l
# attr_sp_mtx = matrix_from_file('./attr_after_encoding',mtx_shape)
# attr_sp_mtx = matrix_from_file('D:\A_LYLI_learning\#LAB\lyl\encoded_actor.csv',mtx_shape)


error('transforming...')
pca = decomposition.RandomizedPCA(n_components = component, whiten=False)
transformed_data = pca.fit_transform(attr_sp_mtx.A)
attr_trans_sp_mtx = sparse.csc_matrix(transformed_data)

write_file('attr_pca_transformed_%d'%component,attr_trans_sp_mtx)

pca_attrs = pd.DataFrame()
pca_attrs[0] = pca.explained_variance_
pca_attrs[1] = pca.explained_variance_ratio_
pca_attrs.columns = ["pca.explained_variance_","pca.explained_variance_ratio_"]
pca_attrs.to_csv('pca_statics_%d.csv'%component,index_label='attribute')
pca_attrs.to_excel('pca_statics_%d.xls'%component,index_label='component')
print 'total_explained_variance_ratio_:',sum(pca.explained_variance_ratio_)

# normalize
mtx_shape = (row,component)
attr_sp_mtx_pca = matrix_from_file('attr_pca_transformed_%d'%component,mtx_shape)
attr_mtx_pca = attr_sp_mtx_pca.A

error('normalize...')
for i in xrange(component):
	attr_mtx_pca[:,i] = normalize_col(attr_mtx_pca[:,i])

#####
attr_nm_sp_mtx_pca = sparse.csc_matrix(attr_mtx_pca)
write_file('attr_pca_%d'%component,attr_nm_sp_mtx_pca)
#####


error('done')






